#coding=utf-8
#!/usr/bin/python

'''
Created on 2012-11-17

@author: Neo.Chen
'''

import os
import sys
sys.path.append("../")
import time
import string
import logging
import threading
from ktlib import ktlib

g_active_count = 0              #当前心跳
g_last_count = 0                #上一次的心跳
g_error_count = 0               #心跳出错的次数
g_string_error_count = 0        #字符串出错的次数
g_bTest = False                 #是否进行测试


def app_restart():
    #使用全局函数
    global g_bTest
    if g_bTest:
        print("app_restart app")
        logging.warning("app_restart app")
    else:
        #重启进程
        os.system("pkill -9 vcs_reclive")
        os.system("pkill -9 ffmpeg")
        os.system("cd /opt/bin")
        os.system("./reclivestart.sh")
        logging.warning("restart app")
        #发邮件通知
        ktlib.send_email("admin@sky-mobi.com", "neo.chen@sky-mobi.com", "recserver restart", "rt")
        ktlib.send_email("admin@sky-mobi.com", "18758166494@139.com", "recserver restart", "rt")
        ktlib.send_email("admin@sky-mobi.com", "tristan.yang@sky-mobi.com", "recserver restart", "rt")
        ktlib.send_email("admin@sky-mobi.com", "13958110295@139.com", "recserver restart", "rt")
        #写单独的日志文件
        f = ktlib.open_file("./reccheck_py.log", "a")
        f.write("check app error!")
        f.close()

def ps_string_check(ps_string):
    #使用全局变量
    global g_string_error_count
    #如果字符串得到0则停止10秒再检测，如果还是有问题，则
    if len(ps_string) == 0:
        g_string_error_count = g_string_error_count + 1
        if g_string_error_count > 3:
            logging.warning("ps_string check failed!")
            app_restart()
    else:
        g_string_error_count = 0


def cpu_check(ps_string):
    cpu = 0
    ps_arr = [""]
    if len(ps_string) > 2:
        ps_arr = ps_string.split(' ')
        index = 0
        for item in ps_arr:
            if len(item) > 2:
                index = index + 1
                if index == 3:
                    #由于字符串不一定是纯数字的，所以要异常捕获，避免崩溃
                    try:
                        cpu = string.atof(item)
                    except:
                        tmp = 0
                        #logging.warning("str to float error!, item :" + item)
                    logging.info("cpu :" + item)
    if cpu > 50:
        logging.warning("cpu check failed! cpu:" + item)
        app_restart()

def rec_check():
    ps_string = os.popen('ps aux|grep vcs_reclive | grep -v sh | grep S','r').read()
    logging.debug("rec_check, ps_string:"  + ps_string)
    #字符串检测
    ps_string_check(ps_string)
    #cpu检测
    cpu_check(ps_string)

def RestartForTime():
    #每天定时重启一次
    str = time.ctime(time.time())
    start = 0
    npos = str.find("00:15:", start)
    if npos != -1:
        logging.warning("time :" + str)
        app_restart()

def active_check():
    #使用全局变量
    global g_active_count
    global g_last_count
    global g_error_count
    #检查心跳包，如果检测3次都没有变化，则认为故障
    logging.info("thread loop, active:" + "%d"%g_active_count + ", last:" + "%d"%g_last_count)
    if g_active_count != g_last_count:
        g_error_count = 0
        g_last_count = g_active_count
    else:
        g_error_count = g_error_count + 1
        if g_error_count > 3:
            logging.warning("active check error!")
            app_restart()
            g_error_count = 0
            g_last_count = g_active_count

def loop(x,y):
    logging.info("thread loop enter")
    while True:
        logging.info("thread loop, check...")
        #每天定时重启一次
        #RestartForTime()
        #心跳包检测
        active_check()
        logging.info("thread loop, check end")
        time.sleep(40)

def main():
    print("main start...")
    logging.basicConfig(level=logging.DEBUG,
                format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                datefmt='%a, %d %b %Y %H:%M:%S',
                filename='myapp.log',
                filemode='w')
    threading.Thread(target=loop, args=(0,1)).start()
    while True:
        logging.info("main, app check...")
        rec_check()
        logging.info("main, app check end")
        #使用全局变量
        global g_active_count
        g_active_count = g_active_count + 1
        logging.info("main, active:" + "%d"%g_active_count)
        time.sleep(10)

if __name__ == '__main__':
    ktlib.ForkMySelf("reccheck", 3)
    try:
        main()
    except:
        ktlib.write_crash_log()

















